﻿/*======================================================================
== Copyright : BlueCurve (c)
== Licence   : Gnu/GPL v2.x
== Author    : Teddy Albina
== Email     : bluecurveteam@gmail.com
== Web site  : http://www.codeplex.com/BlueCurve
========================================================================*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Xml.Linq;

namespace BlueCurve.Common.Ngrams
{
    /// <summary>
    /// Construit les dictionnaires de ngrams
    /// </summary>
    public static class NgramsDictionnaries
    {
        #region 'Configuration'

        /// <summary>
        /// Bibliothèque de fonctions communes
        /// </summary>
        private static CommonFunctions Common = new CommonFunctions();
        /// <summary>
        /// Caractères à supprimer des dictionnaires
        /// </summary>
        public static readonly List<char> stopchar = new List<char>()
        {
                '?', ',', ';', '.', ':', '!', '§', '%', '*', 'µ', '^',
                '|', '-', '[', '(', '{', '\'', '#', '\'', '~', '&', '²',
                '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
                '<', '>', '}', '_', '=', ')', '¨', '$', '£', '¤', '+',
                '/', '\\', ']', '@', '`', '°', '€', '', '', '', '',
                '', '', '', '', '', '', '', '', '¶', '¸'
        };
        /// <summary>
        /// Dictionnaries
        /// </summary>
        public static Dictionary<string, List<string>> m_dictionnaries = new Dictionary<string, List<string>>();

        #endregion
        
        /// <summary>
        /// Génère un dictionnaire de ngrams à partir
        /// d'un fichier texte
        /// </summary>
        /// <param name="path">Chemin d'accès au fichier</param>
        public static void GenerateDictionnaries(string path)
        {
            string[] files = Directory.GetFileSystemEntries(path);
            for (int i = 0; i < files.Length; i++)
                ThreadPool.QueueUserWorkItem(new WaitCallback(Treat), files[i]);
        }
        

        /// <summary>
        /// Fonction de traitement
        /// </summary>
        /// <param name="state">Objet de travail du thread</param>
        private static void Treat(object state)
        {
            string file = state as string;
            string end = Common.GetLastPathElementFromPath(file);
            string bn = Guid.NewGuid().ToString();
            byte[] b = File.ReadAllBytes(file);
            byte[] c = new byte[b.Length];
            for (int i = 0; i < b.Length; i++)
            {
                if (b[i].Equals(BitConverter.GetBytes(0)[0]))
                    c[i] = BitConverter.GetBytes(' ')[0];
                else
                    c[i] = b[i];
            }

            MemoryStream fs = new MemoryStream(c, 0, c.Length);
            byte[] raw = new byte[c.Length];
            fs.Read(raw, 0, c.Length);
            byte[] utf8 = Encoding.Convert(Encoding.Default, Encoding.UTF8, raw);
            fs.Close();
            fs.Dispose();

            // Read
            UTF8Encoding encutf8 = new UTF8Encoding();
            string temp = Common.Stripspaces(Common.AccentFilter(encutf8.GetString(utf8))).ToLower();

            StringBuilder strtemp = new StringBuilder();

            for (int i = 0; i < temp.Length; i++)
            {
                if (temp[i].Equals("") || stopchar.Contains(temp[i]) || char.IsControl(temp[i]) || char.IsDigit(temp[i]) || char.IsNumber(temp[i]) || char.IsPunctuation(temp[i]) && !temp[i].Equals(' '))
                    strtemp.Append(' ');
                else
                    strtemp.Append(temp[i]);
            }

            List<object> xmlelements = new List<object>();
            List<string> tab = NGram.GenerateNGrams(strtemp.ToString().Replace(" ", ""), 3);

            tab.ForEach(
                delegate(string mn)
                {
                    xmlelements.Add(new XElement("words", new XCData(mn)));
                });

            XElement xdoc = new XElement("node",
                new XElement("language", end.Substring(0, end.Length-3).Split('_')[0]), xmlelements.ToArray());
            xdoc.Save(string.Concat(@"Dictionnaries\", end.Split('_')[0], ".xml"));

            c = null;
        }

        
        /// <summary>
        /// Chargement des dictionnaires
        /// </summary>
        public static void LoadDictionnaries()
        {
            string[] files = Directory.GetFileSystemEntries("Dictionnaries");

            for (int i = 0; i < files.Length; i++)
                ThreadPool.QueueUserWorkItem(new WaitCallback(ReadDictionnaries), files[i]);
        }


        /// <summary>
        /// Lit et extrait le contenu des
        /// dictionnaires
        /// </summary>
        /// <param name="state">Objet de travail du thread</param>
        private static void ReadDictionnaries(object state)
        {
            string file = state as string;
            string language = string.Empty;
            string content = string.Empty;

            XDocument doc = XDocument.Load(file);
            var y = from x in doc.Descendants()
                    where x.Name == "language"
                    select x.Value;

            foreach (string n in y)
                language = n;

            if (m_dictionnaries.ContainsKey(language))
                return;
            else
                m_dictionnaries.Add(language, new List<string>());

            var v = from x in doc.Descendants()
                    where x.Name == "words"
                    select x.Value;

            foreach (string w in v)
            {
                if (!m_dictionnaries[language].Contains(w))
                    m_dictionnaries[language].Add(w);
            }
        }
    }
}